# Updated by Wei on 2014/09/27
# Try to learn from this file on how to do the splitting
# Updated by Wei on 20140722
# test the file: /home/vgc/juanr/datasets/prune/stage/bigrams-0.78-25.dyn for Juan 
# Updated by Wei on 20140719
# Help Juan to test the following 2 models:
# /home/vgc/juanr/datasets/prune/bigrams-0.78-50-qb.static
# /home/vgc/juanr/datasets/prune/bigrams-0.78-50-qb.dyn
# Updated by Wei on 20140716
# Help Juan to run the following 4 models:
# /home/vgc/juanr/datasets/prune/bigrams-0.75-qb-gov.dyn
# /home/vgc/juanr/datasets/prune/bigrams-0.75-qb-gov.static
# /home/vgc/juanr/datasets/prune/bigrams-1-qb.dyn
# /home/vgc/juanr/datasets/prune/bigrams-0.54-qb.dyn
# Updated by Wei on 20140714
# Updated by Wei on 20140706
# Main focus: try Juan's bigram stuff
from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
import os
import random
import sys
import time
from sets import Set

# one time generation should be enough
def outputAuxNavigationFile():    
    inputFileList = []
    # "/home/vgc/juanr/datasets/prune/bigrams.dyn"
    # "/home/vgc/juanr/datasets/prune/bigrams.static"
    # inputFileName0 = "/home/vgc/juanr/datasets/prune/bigrams.static"
    # inputFileName0 = "/home/vgc/juanr/datasets/prune/bigrams.dyn"
    # /home/vgc/wei/workspace/NYU_IRTK/data/dynamicUnigramFromWei/documentPostingArrays/gov2/GOV2_DocumentPostingArray_WHOLE_20140702_pt_2DTableFromTOP100Postings.binary
    inputFileList.append(inputFileName0)
    currentDocProcessed = 0 
    for currentInputFileName in inputFileList:
        inputFileHandler = open(currentInputFileName,"rb")
        statinfo = os.stat(currentInputFileName)
        fileSize = statinfo.st_size
        print "currentInputFileName: ",currentInputFileName
        print "file size:",fileSize
        numOfBytesRead = 0
        # print currentDocProcessed,numOfBytesRead
        # inputFileHandler.seek(107874259260)
        
        while numOfBytesRead < fileSize:
            byteString = inputFileHandler.read(4 + 4)
            (docID,numOfPostings) = unpack( "2I", byteString)
            print docID,numOfPostings,currentDocProcessed
            
            
            # exit(1)
            if docID != currentDocProcessed:
                print "----->",docID,currentDocProcessed,numOfPostings,numOfBytesRead
                exit(1)
            
            if docID == 4:
                for i in range(0,numOfPostings):
                    byteString = inputFileHandler.read(4 + 4 + 4 + 4 + 4)
                    (termID,static,dynamic,combined,score) = unpack( "1I4f", byteString)
                    # print "----->",i,termID,static,dynamic,combined,score
                    print "----->",i,termID,static,dynamic,combined,score
                print
                exit(1)
            
            
            if currentDocProcessed % 100000 == 0:
                print currentDocProcessed,numOfBytesRead
            
            currentDocProcessed += 1

            numOfBytesRead += 8 + numOfPostings * 4 * 5
            inputFileHandler.seek(numOfBytesRead)
        inputFileHandler.close()
    print "Ends."

def mainInjectionProcudure(beginningFilePosition,numOfDocumentsNeededToBeProcessed,outputNameSuffix):
    print "Begins..."
    # Updated by Wei on 2014/09/27
    #fileNameFromJuan = "bigrams-xx-10.static"
    fileNameFromJuan = "bigrams-xx-10.dyn"
    
    # attemp5 on 2014/07/19
    # pending
    # "bigrams-0.78-50-qb.static"
    # "bigrams-0.78-50-qb.dyn"
    # "bigrams-0.78-50-qb.static"
    # fileNameFromJuan = "bigrams-0.78-25.dyn"
    # fileNameFromJuan = "bigrams-v-10.dyn"
    # fileNameFromJuan = "bigrams-v-10.static"
    # fileNameFromJuan = "bigrams-v-25.dyn"
    
    # attempt3 on 2014/07/16
    # fileNameFromJuan = "bigrams-0.54-qb.dyn"
    
    
    fileBasePath = "/home/vgc/juanr/"
    fileFromJuan = fileBasePath + fileNameFromJuan
    #fileFromJuan = "/home/vgc/juanr/datasets/prune/bigrams-0.75-qb-gov.static"
    #fileFromJuan = "/home/vgc/juanr/datasets/prune/bigrams-0.75-qb-gov.dyn"
    #fileFromJuan = "/home/vgc/juanr/datasets/prune/bigrams-1-qb.dyn"
    #fileFromJuan = "/home/vgc/juanr/datasets/prune/bigrams-0.54-qb.dyn"
    
    # attempt3 on 2014/07/15
    # dynamic
    # fileFromJuan = "/home/vgc/juanr/datasets/prune/bigrams-1-qb.dyn"
    # static
    # fileFromJuan = "/home/vgc/juanr/datasets/prune/bigrams-1-qb.static"    
    
    # attempt2 on 2014/07/14
    # dynamic
    # fileFromJuan = "/home/vgc/juanr/datasets/prune/bigrams-0.66-1.dyn"
    # static
    # fileFromJuan = "/home/vgc/juanr/datasets/prune/bigrams-0.54-qb.static"

    # attemp1 on 2014/07/06
    # dynamic
    # fileFromJuan = "/home/vgc/juanr/datasets/prune/bigrams-0.66-1.dyn"
    # static
    # fileFromJuan = "/home/vgc/juanr/datasets/prune/bigrams-0.66-1.static"
    fileFromJuanHandler = open(fileFromJuan,"rb")
    statinfo = os.stat(fileFromJuan)
    fileSize = statinfo.st_size
    print "file size from Juan:",fileSize
    print "fileFromJuan:",fileFromJuan
    
    outputFileForJuan = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/" + fileNameFromJuan + "_withImpactScoreAdded_" + str(outputNameSuffix)
    outputFileForJuanHanlder = open(outputFileForJuan,"wb")
    print "outputFileForJuan:",outputFileForJuan
    
    auxFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/GOV2_DocumentPostingArray_WHOLE_20140702_pt_2DTableFromTOP100Postings.binary"
    currentAuxFileName = auxFileName1
    inputFileHandler = open(currentAuxFileName,"rb")
    statinfo = os.stat(currentAuxFileName)
    fileSize = statinfo.st_size
    print "file size from Wei:",fileSize
    
    fileFromJuanHandler.seek(beginningFilePosition)
    inputFileHandler.seek(beginningFilePosition)
    
    numOfBytesRead = 0
    numOfBytesWrite = 0
    numOfDocsProcessed = 0
    # numOfDocsProcessed = 6000000
    
    while numOfBytesRead < fileSize:
        if numOfDocsProcessed % 1000 == 0:
            print "(DEBUG):",numOfDocsProcessed,numOfBytesRead
        # print "numOfBytesRead:",numOfBytesRead
        byteString = inputFileHandler.read(4 + 4)
        (docID,numOfPostings) = unpack( "2I", byteString)
        # print "docIDFromFile1:",str(docID),str(numOfPostings)
        termIDImpactScoreDict = {}
        for index in range(0,numOfPostings):
            byteString = inputFileHandler.read(4 + 4 + 4 + 4 + 4)
            (termID,staticProbability,dynamicProbability,combinedProbability,impactScore) = unpack( "1I4f", byteString)
            
            # print index,termID,staticProbability,dynamicProbability,combinedProbability,impactScore
            if termID not in termIDImpactScoreDict:
                termIDImpactScoreDict[termID] = impactScore
            else:
                print "duplicated termID"
                exit(1)
        
        # print "len(termIDImpactScoreDict):",len(termIDImpactScoreDict)
        # Use termIDImpactScoreDict to fix Juan's impact score problem
        byteString = fileFromJuanHandler.read(4 + 4)
        (docIDFromJuan,numOfPostingsFromJuan) = unpack( "2I", byteString)
        assert docIDFromJuan == docID
        assert numOfPostingsFromJuan == numOfPostings
        # print "docIDFromFile2:",str(docIDFromJuan),str(numOfPostingsFromJuan)
        outputFileForJuanHanlder.write(pack("2I",docIDFromJuan,numOfPostingsFromJuan))
        numOfBytesWrite += 8
        for index in range(0,numOfPostingsFromJuan):
            byteString = fileFromJuanHandler.read(4 + 4 + 4 + 4 + 4)
            (termID,staticProbability,dynamicProbability,combinedProbability,impactScore) = unpack( "1I4f", byteString)
            
            # print index,termID,staticProbability,dynamicProbability,combinedProbability,impactScore
            if termID not in termIDImpactScoreDict:
                print "50000000 problem detected",docID,termID
                impactScore = 0.0
                #print "detail"
                #print "docIDFromJuan:",docIDFromJuan
                #print "numOfPostingsFromJuan:",numOfPostingsFromJuan
                #print "docID:",docID
                #print "numOfPostings:",numOfPostings
                #print "termID missing:",termID
                #exit(1)
            else:
                impactScore = termIDImpactScoreDict[termID]
            
            # print index,termID,staticProbability,dynamicProbability,combinedProbability,impactScore
            outputFileForJuanHanlder.write(pack("1I4f",termID,staticProbability,dynamicProbability,combinedProbability,impactScore))
            numOfBytesWrite += 20
        # print
        numOfBytesRead += 4
        numOfBytesRead += 4
        numOfBytesRead += numOfPostings * 4 * 5
        
        
        numOfDocsProcessed += 1
        
        if numOfDocsProcessed == numOfDocumentsNeededToBeProcessed:
            print numOfDocsProcessed,"docs processed."
            break
              
        #if docID == 100000:
        #    break
        
    print "numOfDocsProcessed:",numOfDocsProcessed
    print "numOfBytesRead:",numOfBytesRead
    print "numOfBytesWrite:",numOfBytesWrite
    inputFileHandler.close()
    
    print "Overall:"
    print "currentAuxFileName:",currentAuxFileName
    print "fileFromJuan:",fileFromJuan
    print "outputFileForJuan:",outputFileForJuan
    print "Ends."

print "Program Begins..."
print "len(sys.argv):",len(sys.argv)
beginningFilePosition = 0
numOfDocumentsNeededToBeProcessed = 1
if len(sys.argv) != 4:
    print "Illegal # of arguments"
    print "Usage: python programName.py fileBeginningPostion # of documents needed to process outputName suffix"
    exit(1)
else:
    beginningFilePosition = int(sys.argv[1])
    numOfDocumentsNeededToBeProcessed = int(sys.argv[2])
    outputNameSuffix = sys.argv[3]
    print "beginningFilePosition:",beginningFilePosition
    print "numOfDocumentsNeededToBeProcessed:",numOfDocumentsNeededToBeProcessed
mainInjectionProcudure(beginningFilePosition,numOfDocumentsNeededToBeProcessed,outputNameSuffix)
print "Program Ends."

